################################
#	John Henderson
#	Gerrymandering Incumbency 
#		(with Brian Hamel and Aaron Goldzimer)
#		January 1, 2018
#
################################
# CD t.test results function
################################

rm(list=ls())
library(stringr)    
library(foreign) 
library(mice)  
library(matrixStats)                                     

setwd('~/Dropbox/StateRedistricting/replication/short')  

state=chamber=NULL
flips=NULL   

imputeNAplans=function(dists,plans=NULL){
	# imputing missing cells for partial plans
	# can't just align districts by number, since these have geographical 
	# definition; and don't have geographical boundaries for non-districts
	# in shapefiles, etc. 
	
	# hard and fast rule: sort districts on the pr(Obama 08) for whole plans, 
	# then match existing districts in partial plans to those that reduce the k-plan
	# distances
	if(is.null(plans)){
		warning('No index for plans is included: care is required to map back onto the various alternatives.')
	}   
	# length of datas
	m=length(dists)
	n=nrow(dists[[1]])  
	# length of completes
	cm=0
	#mu=array(NA,m)  
	is.cm=array(FALSE,m)  
	ord.dists=list() 
	ord.mat=matrix(NA,n,length(dists))
	for(i in 1:m){
		if(all(complete.cases(dists[[i]]))){
			cm=cm+1
			is.cm[i]=T
			mu=dists[[i]][,1]/(dists[[i]][,1]+dists[[i]][,2])
			ord.dists[[i]]=dists[[i]][order(mu),]
			
			ord.mat[,i]=t(t(sort(mu)))
		}
	}   
	
	for(i in 1:m){  
		if(is.cm[i]==F){
			m0=sort(dists[[i]][,1]/(dists[[i]][,1]+dists[[i]][,2]),na.last=T)
			n0=as.numeric(names(m0))
			n1=array(NA,n)
			for(p in 1:length(m0)){ 
				if(is.na(m0[p])){
					break()
				}
				dd=sqrt(rowSums((ord.mat-m0[p])^2,na.rm=T)) 
				if(length(which(!is.na(n1)))>0){
					dd[1:max(which(!is.na(n1)))]=NA
				}
				n1[which(dd==min(dd,na.rm=T))]=n0[p]
				
				#if dd object is approaching length(m0) *too fast*
				if(p<length(m0[!is.na(m0)]) & max(which(is.na(n1)))<length(m0)){
					n1=c(n1[-c(max(which(is.na(n1))))],NA)
				} 			      			
			}
			n1[which(is.na(n1))]=sample(n0[which(is.na(m0))],replace=F)	
			ord.dists[[i]]=dists[[i]][n1,]		   		
		}
	}
	 
	# quasi-randomization simulation : plans that have missing districts
	#  will have missings imputed for *all* the other plans that have those 
	#  districts : average district bias will be recorded ... 
	# complete plans are analyzed as is
	dist_insa=matrix(NA,n,1)
	dist_ins=matrix(NA,n,m) 
	colnames(dist_ins)=plans
	for(i in 1:m){  
		if(is.cm[i]==F){
			qn=ord.dists[[i]][,1]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])-ord.dists[[i]][,2]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])			
		  	mat_temp=matrix(qn,n,sum(is.cm))
			jj=0
			for(j in which(is.cm)){
				jj=jj+1
				qs=ord.dists[[j]][,1]/(ord.dists[[j]][,1]+ord.dists[[j]][,2])-ord.dists[[j]][,2]/(ord.dists[[j]][,1]+ord.dists[[j]][,2])							
				mat_temp[which(is.na(qn)),jj]=qs[which(is.na(qn))]				
			}                                                     
			colnames(mat_temp)=paste(plans[i],'sim',1:jj,sep='_')
			if(mean(complete.cases(dist_insa))==0){
				dist_insa=mat_temp
			} else {
				dist_insa=cbind(dist_insa,mat_temp)
			}
		} else {
			dist_ins[,i]=ord.dists[[i]][,1]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])-ord.dists[[i]][,2]/(ord.dists[[i]][,1]+ord.dists[[i]][,2])			
		}
	} 
	if(mean(is.cm)<1){
		dist_ins=cbind(dist_ins[,which(is.cm==T)],dist_insa)
	}
	return(dist_ins) 
}
 
############################
#
# counterfactual maps 2010
#
############################
                          
states=c(
  "AL","AK","AZ","AR","CA",  "CO","CT","DE","FL",  "GA","HI","ID","IL","IN",  "IA","KS","KY","LA","ME",  "MT","NE","NV","NH","NJ", "OR",  
  "NM","NY","NC","ND","OH",  "OK","MD","MA","MI",  "MN","MS","MO","PA","RI",  "SC","SD","TN","TX","UT",  "VT","VA","WA","WV","WI", "WY")

states=sort(states)
     
plans=read.csv('redist_authority.csv',stringsAsFactors=F)    
plans10=plans[which(plans[,1]==2010),]
included=array(FALSE,nrow(plans10))
for(i in 1:length(states)){
	included[which(states[i]==plans10[,2])]=TRUE
}

plans10=plans10[included,]
plans10=plans10[order(plans10[,2]),]

plansCD10 = array(NA,length(states))
plansCD10[which(plans10$congress_court==1)]='C'
plansCD10[which(plans10$congress_independent==1)]='I'
plansCD10[which(plans10$congress_politician==1)]='B'
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='nonpartisan')]='B' # NE
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='split')]='B'
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='D')]='D'
plansCD10[which(is.na(plansCD10) & plans10$congress_legislatue_partycontrol=='R')]='R'
plansCD10[which(is.na(plansCD10))]='N'

plansAD10 = array(NA,length(states))
plansAD10[which(plans10$state_court==1)]='C'
plansAD10[which(plans10$state_independent==1)]='I'
plansAD10[which(plans10$state_politician==1)]='B'
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='nonpartisan')]='B' # NE
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='split')]='B'
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='D')]='D'
plansAD10[which(is.na(plansAD10) & plans10$state_legislature_partycontrol=='R')]='R'
plansAD10[which(is.na(plansAD10))]='N'
 
plansSD10=plansAD10
                
states_cf=c('AK','AZ','CA','CO','FL','FL','ID','MT','NC','NC','NM','NV','OH','SC','TX','VA','WA')    
ix=c(which(states=='AK'),
	which(states=='AZ'),
	which(states=='CA'),
	which(states=='CO'),	
	which(states=='FL'),
	which(states=='FL'),  
	which(states=='ID'), 
	which(states=='MT'),  
	which(states=='NC'),
	which(states=='NC'),	 
	which(states=='NM'),
	which(states=='NV'),	 	
	which(states=='OH'),	
	which(states=='SC'),
	which(states=='TX'),
	which(states=='VA'),
	which(states=='WA')) 

plansCD10=plansCD10[ix] 
plansCD10[which(states_cf=='AK')]='I'
plansCD10[which(states_cf=='MT')]='I'
plansCD10[which(states_cf=='ID')]='I'
plansCD10[which(states_cf=='FL')[2]]='C'

#'CA'
#'CO'     
#plansCD10[ix]
#plansAD10[ix]

#################
# cd
#################

states_ix=states_cf[c(which(plansCD10=='D'),which(plansCD10=='R'),which(plansCD10=='B'),which(plansCD10=='C'),which(plansCD10=='I'))]  
pl10=plansCD10[c(which(plansCD10=='D'),which(plansCD10=='R'),which(plansCD10=='B'),which(plansCD10=='C'),which(plansCD10=='I'))]  

e08=read.table('sims2010/cd.txt')
nms=e08[1,]
e08=as.data.frame(e08[-c(1),])
names(e08)=t(t(nms))

e08$mcnshare=as.numeric(as.character(e08[,10]))/(as.numeric(as.character(e08[,9]))+as.numeric(as.character(e08[,10])))                     
e08$dif=abs((1-e08[,11])-e08[,11])      

election08=tapply(e08$dif,e08[,1],mean)
election08=election08[-c(which(names(election08)=='st'))]

sims=matrix(NA,length(states),211)
for(j in 1:length(states)){     

	fls=paste('sims2010/',states[j],'/plans/',length(which(e08[,1]==states[j])),'.txt',sep='')   
	if(file.exists(fls)){
		cds=read.table(fls)[,-c(1)]        
	} else if(!file.exists(fls)){
		fla=fls                        
		pp=0
		while(!file.exists(fla) & pp<60){
			pp=pp+1
			fla=paste('sims2010/',states[j],'/plans/',pp,'.txt',sep='')   					
		}
		fls=fla 
		if(file.exists(fls)){ 
			cds=read.table(fls)[,-c(1)]     
		} else{
			next(paste(states[j],'has no simulations',sep=' '))
		}
	}          

	sims[j,1]=states[j] 
	if(length(which(e08[,1]==states[j]))>1){
		sims[j,-c(1)][1:dim(cds)[1]]=rowMeans(abs((1-cds)-cds))
	} else{
		sims[j,-c(1)][1:length(cds)]=(abs((1-cds)-cds))
	}
}

sims=as.data.frame(sims[,1:201])
for(j in 2:ncol(sims)){
	sims[,j]=as.numeric(as.character(sims[,j]))
}


xx=table(e08[,1])
for(i in 1:length(xx)){
	if(xx[i]==1){
		sims[which(names(xx)[i]==sims[,1]),-c(1)]=0  
		election08[which(names(xx)[i]==names(election08))]=0
	}
}

is.even <- function(x) x %% 2 == 0 

#############################  
## Florida has multiple plans and so need to adjust for the court map in 2015/2016
#############################
#folders=paste('CD_plans/',system('ls ~/Dropbox/StateRedistricting/Data/plans2010/FL/CD/CD_plans/',intern=T),sep='')
#full_plans=read.csv('~/Dropbox/StateRedistricting/Data/plans2010/FL/scraped/PlanDataIns.csv',header=F,stringsAsFactors=F)
#dates=array(NA,length(folders))
#folders=gsub(folders,pattern='CD_plans/',replace='') 
#for(i in 2:length(folders)){
#	ix=which(gsub(folders[i],pattern='_adopted',replace='')==full_plans[,1])
#	dates[i]=str_sub(full_plans[ix,3],7)
#}
#dates[1]='2002'
#save(dates,folders,file='~/Dropbox/StateRedistricting/replication/short/CD_FL_dates.Rdata')
load('plans2010/CD_FL_dates.Rdata')  
#############################
 
######################
# flips to do here
###################### 

load('voteData.Rdata')
load('stateSwings.Rdata')

iq=0
cnts=length(states_ix)+1 
nls=fls=0  

propGT=matrix(NA,length(states_ix),4)   

kk=c()         
ss=c()
ii=c()     

ss.dif=c()
ss.mu=c()
source('analysisSTFun.R')    

for (state in states_ix){  
	iq=iq+1;
	cnts=cnts-1
    chamber='CD'
	#source('~/Dropbox/StateRedistricting/Code/plans2010/analysisTest.R')  
	
	if(state!='AK' & state!='OH' & state!='MT'){ 
		st=analysisST(state=state,chamber=chamber,flips=flips,m0=1000,m1=1000000)	
		k_get=st$k_get
		adopted=st$adopted
		folders=st$folders		   
		app=ap=st$ap
		opp=op=st$op
	}
	 
	
	k=ncol(op)
	m=nrow(op)
	k_get=array(NA,k)
	
	if(length(dim(ap)[2])==0){
       for(j in 1:k){
			k_get[j]=mean(abs(ap)>abs(op[,j]),na.rm=T)
		}		
		#k_get=colMeans(abs(ap)>abs(op))
	} else if(length(dim(ap)[2])>0 & state=='FL' & fls==0){
		folds=folders[which(as.numeric(dates)<2015)]
		for(j in 1:dim(ap)[2]){
			if(length(which(colnames(ap)[j]==folds))){
				break()
			}
		} 		
		app=ap=ap[,j]
		
		for(j in 1:k){
			k_get[j]=mean(abs(ap)>abs(op[,j]),na.rm=T)
		}
		#k_get=colMeans(abs(ap)>abs(op)) 
		#fls=fls+1      	
	} else if(length(dim(ap)[2])>0 & state=='FL' & fls==1){ 
		folds=folders[which(as.numeric(dates)>2014)]
		for(j in 1:dim(ap)[2]){
			if(length(which(colnames(ap)[j]==folds))){
				break()
			}
		} 		
		app=ap=ap[,j]
		
		for(j in 1:k){
			k_get[j]=mean(abs(ap)>abs(op[,j]),na.rm=T)
		}
		#k_get=colMeans(abs(ap)>abs(op)) 
		#fls=fls-1	
	} else if(length(dim(ap)[2])>0 & state=='NC' & nls==0){
		#folds=folders[which(as.numeric(dates)<2015)]
		#for(j in 1:dim(ap)[2]){
		#	if(length(which(colnames(ap)[j]==folds))){
		#		break()
		#	}
		#} 		
		app=ap=ap[,1]
		
		for(j in 1:k){
			k_get[j]=mean(abs(ap)>abs(op[,j]),na.rm=T)
		}
		#k_get=colMeans(abs(ap)>abs(op)) 
		#fls=fls+1      	
	} else if(length(dim(ap)[2])>0 & state=='NC' & nls==1){ 
		#folds=folders[which(as.numeric(dates)>2014)]
		#for(j in 1:dim(ap)[2]){
		#	if(length(which(colnames(ap)[j]==folds))){
		#		break()
		#	}
		#} 		
		app=ap=ap[,2]
		
		for(j in 1:k){
			k_get[j]=mean(abs(ap)>abs(op[,j]),na.rm=T)
		}
		#k_get=colMeans(abs(ap)>abs(op)) 
		#fls=fls-1	
	} else {
	  	for(j in 1:k){
			k_get[j]=mean(abs(ap)>abs(op[,j]),na.rm=T)
		}		
	}    
	
               	  
	kk=c(kk,k_get)
	ss=c(ss,rep(length(k_get),length(k_get))) 
	ii=c(ii,rep(pl10[iq],length(k_get))) 

	if(state!='AK' & state!='OH' & state!='MT'){ 	
		propGT[iq,1]=mean(k_get,na.rm=T)
	} else {
		 #propGT[iq,1]=state
	}
	
	if(state!='AK' & state!='OH' & state!='MT'){ 
		st=analysisST(state=state,chamber=chamber,flips=flips,m0=1000,m1=1000000)	
		k_get=st$k_get
		adopted=st$adopted
		folders=st$folders		   
		ap=st$ap
		op=st$op
	}	
	#source('~/Dropbox/StateRedistricting/Code/plans2010/analysisST.R')
    # adopted & k_get are the relevant objects  
	names(k_get)[adopted]=paste('adopted',names(k_get)[adopted],sep='_')
	
	sts=state
	if(state=='FL' & fls==0){
		folds=folders[which(as.numeric(dates)<2015)] 
		inc=c()
		for(i in 1:length(folds)){
			inc=c(inc,grep(names(k_get),pattern=folds[i]))  
		}
		k_get=k_get[sort(inc)]		
		fls=fls+1
		adopted=grep(names(k_get),pattern='adopted') 		
		#sts=paste(sts,'\n(2011)',sep='')            
		yar=2011
	} else if(state=='FL' & fls==1){
		folds=folders[which(as.numeric(dates)>2014)]
		inc=c()
		for(i in 1:length(folds)){
			inc=c(inc,grep(names(k_get),pattern=folds[i]))  
		}
		k_get=k_get[sort(inc)]	  
		adopted=grep(names(k_get),pattern='adopted') 
		#sts=paste(sts,'\n(2015)',sep='')	 
		yar=2015
	} 

	if(state=='NC' & nls==0){
		k_get=k_get[-c(adopted[2])]
		adopted=grep(names(k_get),pattern='adopted')
		yar=2011  
		nls=nls+1       	 
	} else if(state=='NC' & nls==1){  
		k_get=k_get[-c(adopted[1])]
		adopted=grep(names(k_get),pattern='adopted')
		yar=2016		
	}
   		
	# remove those with extra but simular adopted plans
	if(length(grep(names(k_get),pattern='adopted'))>1){
		names(k_get)[grep(names(k_get),pattern='adopted')[-c(1)]]=gsub(names(k_get)[grep(names(k_get),pattern='adopted')[-c(1)]],pattern='adopted_',replace='')
		adopted=adopted[1]
	}
	    
   
 
	#m=length(adopted)
	avg.sims=2
	if(avg.sims==1){ # average over partial plans 
		kk_get=avgSims(k_get,discard=F)
	} else if(avg.sims==2){ # retain all partial plans
		kk_get=k_get
	} else if(avg.sims==3){ # discard all partial plans 
		kk_get=avgSims(k_get,discard=T)		
	}        
   	

	if(state!='AK' & state!='OH' & state!='MT'){ 
		propGT[iq,2]=mean(kk_get[c(grep(names(kk_get),pattern='adopted'))]>kk_get[-c(grep(names(kk_get),pattern='adopted'))])
	} else {
		 #propGT[iq,2]=state
	}     
	if(length(kk_get)>0){
		mu=mean(kk_get,na.rm=T)
		sds=sd(kk_get,na.rm=T) 
		k.means.sim=((kk_get-mu)/sds)
	    #k.mean.10=((mean.10-mu)/sds)	
		#s.dif=mean(s.mean.10-s.means.sim)
		
		ss.dif=c(ss.dif,mean(k.means.sim[c(grep(names(k.means.sim),pattern='adopted'))])-mean(k.means.sim[-c(grep(names(k.means.sim),pattern='adopted'))]))
		ss.mu=c(ss.mu,mean(kk_get[c(grep(names(kk_get),pattern='adopt'))],na.rm=T)-mean(kk_get[-c(grep(names(kk_get),pattern='adopt'))],na.rm=T))
	}
	if(state!='AK' & state!='OH' & state!='MT'){  
		propGT[iq,3]=mean(abs(app))-mean(abs(opp))
		propGT[iq,4]=mean(mean(abs(app))>(colMeans(abs(opp))))
	}
}

#propGT=(cbind(pl10,propGT))
         
# win margin for non court plans overall
mean(propGT[pl10!='C',3]) == #1.2 percentage points
mean(propGT[pl10!='C',2]>.95,na.rm=T) #.33
median(propGT[pl10!='C',2],na.rm=T) #.55
median(propGT[pl10!='C',1],na.rm=T) 

propGT[c(which(states_ix=='FL')[1],which(states_ix=='NC')[1],which(states_ix=='SC')),3]*100
#0.3983884 6.9653059 0.3599664
propGT[c(which(states_ix=='FL')[1],which(states_ix=='NC')[1],which(states_ix=='SC')),4]*100

propGT[c(which(states_ix=='AZ')[1],which(states_ix=='CA'),which(states_ix=='WA')),1]
propGT[c(which(states_ix=='AZ')[1],which(states_ix=='CA'),which(states_ix=='WA')),3]    
#0.0096345291 0.0003543955 0.0015836391

#y=c(t(kk))   
#ss=as.factor(ss)
#ii=as.factor(ii)
#ii_mat=matrix(0,length(ii),3)
#ii_mat[which(ii=='B'),1]=1
#ii_mat[which(ii=='C'),2]=1
#ii_mat[which(ii=='R'),3]=1 
#colnames(ii_mat)=c('B','C','R')
#summary(lm(y~ss+ii_mat))
      
#t.test(propGT[pl10=='I',1],c(propGT[pl10=='B',1],propGT[pl10=='D',1],propGT[pl10=='R',1]))
t.test(propGT[pl10=='I',4],c(propGT[pl10=='B',4],propGT[pl10=='D',4],propGT[pl10=='R',4]))  
#t.test(propGT[pl10=='I',2],c(propGT[pl10=='B',2],propGT[pl10=='D',2],propGT[pl10=='R',2]))                                                                                          
#t.test(c(propGT[pl10=='I',1],propGT[pl10=='I',2]),c(propGT[pl10=='B',1],propGT[pl10=='D',1],propGT[pl10=='R',1],propGT[pl10=='B',2],propGT[pl10=='D',2],propGT[pl10=='R',2]))
#t.test(propGT[pl10=='I',2],c(propGT[pl10=='B',2],propGT[pl10=='D',2],propGT[pl10=='R',2])) 
#t.test(propGT[pl10=='I',4],c(propGT[pl10=='B',4],propGT[pl10=='D',4],propGT[pl10=='R',4]))                                                                                              

#t.test(propGT[pl10=='C',1],c(propGT[pl10=='B',1],propGT[pl10=='D',1],propGT[pl10=='R',1]))
#t.test(propGT[pl10=='C',2],c(propGT[pl10=='B',2],propGT[pl10=='D',2],propGT[pl10=='R',2]))
t.test(c(propGT[pl10=='C',1],propGT[pl10=='C',2]),c(propGT[pl10=='B',1],propGT[pl10=='D',1],propGT[pl10=='R',1],propGT[pl10=='B',2],propGT[pl10=='D',2],propGT[pl10=='R',2]))       
 
#t.test(propGT[pl10=='I',1],c(propGT[pl10=='C',1]))
#t.test(propGT[pl10=='I',2],c(propGT[pl10=='C',2]))
#t.test(c(propGT[pl10=='I',1],propGT[pl10=='I',2]),c(propGT[pl10=='C',1],propGT[pl10=='C',2]))
                                                                                                                                                                                   
#end